{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Remote data access using pandas"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The pandas library enables access to data displayed on websites using the `read_html()` function and access to the API endpoints of various data providers through the related `pandas-datareader` library."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:14:54.455404Z",
     "start_time": "2018-12-25T19:14:54.447513Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas_datareader.data as web\n",
    "from datetime import datetime\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Download html table with SP500 constituents"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The download of the content of one or more html tables works as follows, for instance for the constituents of the S&P500 index from Wikipedia"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:14:55.680031Z",
     "start_time": "2018-12-25T19:14:54.460128Z"
    }
   },
   "outputs": [],
   "source": [
    "sp_url = 'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies'\n",
    "sp500_constituents = pd.read_html(sp_url, header=0)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:14:55.685673Z",
     "start_time": "2018-12-25T19:14:55.681237Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 505 entries, 0 to 504\n",
      "Data columns (total 9 columns):\n",
      "Security                 505 non-null object\n",
      "Symbol                   505 non-null object\n",
      "SEC filings              505 non-null object\n",
      "GICS Sector              505 non-null object\n",
      "GICS Sub Industry        505 non-null object\n",
      "Headquarters Location    505 non-null object\n",
      "Date first added         402 non-null object\n",
      "CIK                      505 non-null int64\n",
      "Founded                  172 non-null object\n",
      "dtypes: int64(1), object(8)\n",
      "memory usage: 35.6+ KB\n"
     ]
    }
   ],
   "source": [
    "sp500_constituents.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:14:55.702503Z",
     "start_time": "2018-12-25T19:14:55.686865Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Security</th>\n",
       "      <th>Symbol</th>\n",
       "      <th>SEC filings</th>\n",
       "      <th>GICS Sector</th>\n",
       "      <th>GICS Sub Industry</th>\n",
       "      <th>Headquarters Location</th>\n",
       "      <th>Date first added</th>\n",
       "      <th>CIK</th>\n",
       "      <th>Founded</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3M Company</td>\n",
       "      <td>MMM</td>\n",
       "      <td>reports</td>\n",
       "      <td>Industrials</td>\n",
       "      <td>Industrial Conglomerates</td>\n",
       "      <td>St. Paul, Minnesota</td>\n",
       "      <td>NaN</td>\n",
       "      <td>66740</td>\n",
       "      <td>1902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Abbott Laboratories</td>\n",
       "      <td>ABT</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Health Care Equipment</td>\n",
       "      <td>North Chicago, Illinois</td>\n",
       "      <td>1964-03-31</td>\n",
       "      <td>1800</td>\n",
       "      <td>1888</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AbbVie Inc.</td>\n",
       "      <td>ABBV</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Pharmaceuticals</td>\n",
       "      <td>North Chicago, Illinois</td>\n",
       "      <td>2012-12-31</td>\n",
       "      <td>1551152</td>\n",
       "      <td>2013 (1888)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ABIOMED Inc</td>\n",
       "      <td>ABMD</td>\n",
       "      <td>reports</td>\n",
       "      <td>Health Care</td>\n",
       "      <td>Health Care Equipment</td>\n",
       "      <td>Danvers, Massachusetts</td>\n",
       "      <td>2018-05-31</td>\n",
       "      <td>815094</td>\n",
       "      <td>1981</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Accenture plc</td>\n",
       "      <td>ACN</td>\n",
       "      <td>reports</td>\n",
       "      <td>Information Technology</td>\n",
       "      <td>IT Consulting &amp; Other Services</td>\n",
       "      <td>Dublin, Ireland</td>\n",
       "      <td>2011-07-06</td>\n",
       "      <td>1467373</td>\n",
       "      <td>1989</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Security Symbol SEC filings             GICS Sector  \\\n",
       "0           3M Company    MMM     reports             Industrials   \n",
       "1  Abbott Laboratories    ABT     reports             Health Care   \n",
       "2          AbbVie Inc.   ABBV     reports             Health Care   \n",
       "3          ABIOMED Inc   ABMD     reports             Health Care   \n",
       "4        Accenture plc    ACN     reports  Information Technology   \n",
       "\n",
       "                GICS Sub Industry    Headquarters Location Date first added  \\\n",
       "0        Industrial Conglomerates      St. Paul, Minnesota              NaN   \n",
       "1           Health Care Equipment  North Chicago, Illinois       1964-03-31   \n",
       "2                 Pharmaceuticals  North Chicago, Illinois       2012-12-31   \n",
       "3           Health Care Equipment   Danvers, Massachusetts       2018-05-31   \n",
       "4  IT Consulting & Other Services          Dublin, Ireland       2011-07-06   \n",
       "\n",
       "       CIK      Founded  \n",
       "0    66740         1902  \n",
       "1     1800         1888  \n",
       "2  1551152  2013 (1888)  \n",
       "3   815094         1981  \n",
       "4  1467373         1989  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sp500_constituents.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## pandas-datareader for Market Data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`pandas` used to facilitate access to data providers' APIs directly, but this functionality has moved to the related pandas-datareader library. The stability of the APIs varies with provider policies, and as of June 2o18 at version 0.7, the following sources are available"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "See [documentation](https://pandas-datareader.readthedocs.io/en/latest/); functionality frequently changes as underlying provider APIs evolve."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Yahoo Finance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:14:56.812409Z",
     "start_time": "2018-12-25T19:14:55.703363Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 856 entries, 2014-01-02 to 2017-05-25\n",
      "Data columns (total 6 columns):\n",
      "High         856 non-null float64\n",
      "Low          856 non-null float64\n",
      "Open         856 non-null float64\n",
      "Close        856 non-null float64\n",
      "Volume       856 non-null int64\n",
      "Adj Close    856 non-null float64\n",
      "dtypes: float64(5), int64(1)\n",
      "memory usage: 46.8 KB\n"
     ]
    }
   ],
   "source": [
    "start = '2014'\n",
    "end = datetime(2017, 5, 24)\n",
    "\n",
    "yahoo= web.DataReader('FB', 'yahoo', start=start, end=end)\n",
    "yahoo.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IEX"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "IEX is an alternative exchange started in response to the HFT controversy and portrayed in Michael Lewis' controversial Flash Boys. It aims to slow down the speed of trading to create a more level playing field and has been growing rapidly since launch in 2016 while still small with a market share of around 2.5% in June 2018."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:14:57.093439Z",
     "start_time": "2018-12-25T19:14:56.813719Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 1054 entries, 2015-02-09 to 2019-04-16\n",
      "Data columns (total 5 columns):\n",
      "open      1054 non-null float64\n",
      "high      1054 non-null float64\n",
      "low       1054 non-null float64\n",
      "close     1054 non-null float64\n",
      "volume    1054 non-null int64\n",
      "dtypes: float64(4), int64(1)\n",
      "memory usage: 49.4+ KB\n"
     ]
    }
   ],
   "source": [
    "start = datetime(2015, 2, 9)\n",
    "# end = datetime(2017, 5, 24)\n",
    "\n",
    "iex = web.DataReader('FB', 'iex', start)\n",
    "iex.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:14:57.108760Z",
     "start_time": "2018-12-25T19:14:57.095941Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>open</th>\n",
       "      <th>high</th>\n",
       "      <th>low</th>\n",
       "      <th>close</th>\n",
       "      <th>volume</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>date</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2019-04-10</th>\n",
       "      <td>178.18</td>\n",
       "      <td>178.79</td>\n",
       "      <td>176.54</td>\n",
       "      <td>177.82</td>\n",
       "      <td>11701479</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-04-11</th>\n",
       "      <td>178.24</td>\n",
       "      <td>178.40</td>\n",
       "      <td>177.00</td>\n",
       "      <td>177.51</td>\n",
       "      <td>8070967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-04-12</th>\n",
       "      <td>178.00</td>\n",
       "      <td>179.63</td>\n",
       "      <td>177.95</td>\n",
       "      <td>179.10</td>\n",
       "      <td>12329812</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-04-15</th>\n",
       "      <td>178.50</td>\n",
       "      <td>180.50</td>\n",
       "      <td>176.87</td>\n",
       "      <td>179.65</td>\n",
       "      <td>10834762</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2019-04-16</th>\n",
       "      <td>179.00</td>\n",
       "      <td>180.17</td>\n",
       "      <td>178.30</td>\n",
       "      <td>178.87</td>\n",
       "      <td>11215193</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              open    high     low   close    volume\n",
       "date                                                \n",
       "2019-04-10  178.18  178.79  176.54  177.82  11701479\n",
       "2019-04-11  178.24  178.40  177.00  177.51   8070967\n",
       "2019-04-12  178.00  179.63  177.95  179.10  12329812\n",
       "2019-04-15  178.50  180.50  176.87  179.65  10834762\n",
       "2019-04-16  179.00  180.17  178.30  178.87  11215193"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "iex.tail()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Book Data\n",
    "\n",
    "In addition to historical EOD price and volume data, IEX provides real-time depth of book quotations that offer an aggregated size of orders by price and side. This service also includes last trade price and size information.\n",
    "\n",
    "DEEP is used to receive real-time depth of book quotations direct from IEX. The depth of book quotations received via DEEP provide an aggregated size of resting displayed orders at a price and side, and do not indicate the size or number of individual orders at any price level. Non-displayed orders and non-displayed portions of reserve orders are not represented in DEEP.\n",
    "\n",
    "DEEP also provides last trade price and size information. Trades resulting from either displayed or non-displayed orders matching on IEX will be reported. Routed executions will not be reported."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Only works on trading days."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:14:57.278095Z",
     "start_time": "2018-12-25T19:14:57.110743Z"
    }
   },
   "outputs": [],
   "source": [
    "book = web.get_iex_book('AAPL')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:14:57.302122Z",
     "start_time": "2018-12-25T19:14:57.289530Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['symbol',\n",
       " 'marketPercent',\n",
       " 'volume',\n",
       " 'lastSalePrice',\n",
       " 'lastSaleSize',\n",
       " 'lastSaleTime',\n",
       " 'lastUpdated',\n",
       " 'bids',\n",
       " 'asks',\n",
       " 'systemEvent',\n",
       " 'tradingStatus',\n",
       " 'opHaltStatus',\n",
       " 'ssrStatus',\n",
       " 'securityEvent',\n",
       " 'trades',\n",
       " 'tradeBreaks']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(book.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:14:57.318139Z",
     "start_time": "2018-12-25T19:14:57.307227Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>side</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [side]\n",
       "Index: []"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "orders = pd.concat([pd.DataFrame(book[side]).assign(side=side) for side in ['bids', 'asks']])\n",
    "orders.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:14:57.333155Z",
     "start_time": "2018-12-25T19:14:57.319823Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "symbol\n",
      "AAPL\n",
      "\n",
      "marketPercent\n",
      "0.03324\n",
      "\n",
      "volume\n",
      "977659\n",
      "\n",
      "lastSalePrice\n",
      "203.19\n",
      "\n",
      "lastSaleSize\n",
      "3\n",
      "\n",
      "lastSaleTime\n",
      "1555531318248\n",
      "\n",
      "lastUpdated\n",
      "1555532174025\n",
      "\n",
      "bids\n",
      "Empty DataFrame\n",
      "Columns: []\n",
      "Index: []\n",
      "\n",
      "asks\n",
      "Empty DataFrame\n",
      "Columns: []\n",
      "Index: []\n",
      "\n",
      "systemEvent\n",
      "{'systemEvent': 'C', 'timestamp': 1555535400001}\n",
      "\n",
      "tradingStatus\n",
      "{'status': 'T', 'reason': '    ', 'timestamp': 1555500532036}\n",
      "\n",
      "opHaltStatus\n",
      "{'isHalted': False, 'timestamp': 1555500532036}\n",
      "\n",
      "ssrStatus\n",
      "{'isSSR': False, 'detail': ' ', 'timestamp': 1555500532036}\n",
      "\n",
      "securityEvent\n",
      "{'securityEvent': 'MarketClose', 'timestamp': 1555531200000}\n",
      "\n",
      "trades\n",
      "    isISO  isOddLot  isOutsideRegularHours  isSinglePriceCross  \\\n",
      "0    True      True                   True               False   \n",
      "1   False     False                  False               False   \n",
      "2    True     False                  False               False   \n",
      "3   False     False                  False               False   \n",
      "4    True     False                  False               False   \n",
      "5   False     False                  False               False   \n",
      "6   False     False                  False               False   \n",
      "7    True     False                  False               False   \n",
      "8    True     False                  False               False   \n",
      "9    True     False                  False               False   \n",
      "10  False      True                  False               False   \n",
      "11  False     False                  False               False   \n",
      "12   True      True                  False               False   \n",
      "13  False     False                  False               False   \n",
      "14  False     False                  False               False   \n",
      "15  False     False                  False               False   \n",
      "16   True      True                  False               False   \n",
      "17   True     False                  False               False   \n",
      "18   True      True                  False               False   \n",
      "19  False     False                  False               False   \n",
      "\n",
      "    isTradeThroughExempt    price  size      timestamp    tradeId  \n",
      "0                  False  203.190     3  1555531318248  891604355  \n",
      "1                  False  203.200   100  1555531197248  890272160  \n",
      "2                  False  203.140   100  1555531195857  889977488  \n",
      "3                  False  203.155   100  1555531195465  889859713  \n",
      "4                  False  203.140   100  1555531195292  889808657  \n",
      "5                  False  203.110   100  1555531194852  889619571  \n",
      "6                  False  203.100   100  1555531194822  889606541  \n",
      "7                  False  203.180   100  1555531194794  889585630  \n",
      "8                   True  203.180   100  1555531194794  889585556  \n",
      "9                  False  203.180   100  1555531191476  889002606  \n",
      "10                 False  203.190    25  1555531190110  888694678  \n",
      "11                 False  203.140   100  1555531187948  888262843  \n",
      "12                 False  203.160    64  1555531187684  888228377  \n",
      "13                 False  203.160   100  1555531187682  888227956  \n",
      "14                 False  203.160   100  1555531187681  888227469  \n",
      "15                 False  203.160   100  1555531187680  888227241  \n",
      "16                 False  203.160    36  1555531187680  888227079  \n",
      "17                 False  203.160   100  1555531187680  888227061  \n",
      "18                 False  203.170     9  1555531185364  887813837  \n",
      "19                 False  203.150   200  1555531184053  887531882  \n",
      "\n",
      "tradeBreaks\n",
      "Empty DataFrame\n",
      "Columns: []\n",
      "Index: []\n"
     ]
    }
   ],
   "source": [
    "for key in book.keys():\n",
    "    try:\n",
    "        print(f'\\n{key}')\n",
    "        print(pd.DataFrame(book[key]))\n",
    "    except:\n",
    "        print(book[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:14:57.420316Z",
     "start_time": "2018-12-25T19:14:57.334612Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>isISO</th>\n",
       "      <th>isOddLot</th>\n",
       "      <th>isOutsideRegularHours</th>\n",
       "      <th>isSinglePriceCross</th>\n",
       "      <th>isTradeThroughExempt</th>\n",
       "      <th>price</th>\n",
       "      <th>size</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>tradeId</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>203.190</td>\n",
       "      <td>3</td>\n",
       "      <td>1555531318248</td>\n",
       "      <td>891604355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>203.200</td>\n",
       "      <td>100</td>\n",
       "      <td>1555531197248</td>\n",
       "      <td>890272160</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>203.140</td>\n",
       "      <td>100</td>\n",
       "      <td>1555531195857</td>\n",
       "      <td>889977488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>203.155</td>\n",
       "      <td>100</td>\n",
       "      <td>1555531195465</td>\n",
       "      <td>889859713</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>203.140</td>\n",
       "      <td>100</td>\n",
       "      <td>1555531195292</td>\n",
       "      <td>889808657</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   isISO  isOddLot  isOutsideRegularHours  isSinglePriceCross  \\\n",
       "0   True      True                   True               False   \n",
       "1  False     False                  False               False   \n",
       "2   True     False                  False               False   \n",
       "3  False     False                  False               False   \n",
       "4   True     False                  False               False   \n",
       "\n",
       "   isTradeThroughExempt    price  size      timestamp    tradeId  \n",
       "0                 False  203.190     3  1555531318248  891604355  \n",
       "1                 False  203.200   100  1555531197248  890272160  \n",
       "2                 False  203.140   100  1555531195857  889977488  \n",
       "3                 False  203.155   100  1555531195465  889859713  \n",
       "4                 False  203.140   100  1555531195292  889808657  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(book['trades']).head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Quandl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:15:07.500237Z",
     "start_time": "2018-12-25T19:15:06.978720Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 813 entries, 2018-03-27 to 2015-01-02\n",
      "Data columns (total 12 columns):\n",
      "Open          813 non-null float64\n",
      "High          813 non-null float64\n",
      "Low           813 non-null float64\n",
      "Close         813 non-null float64\n",
      "Volume        813 non-null float64\n",
      "ExDividend    813 non-null float64\n",
      "SplitRatio    813 non-null float64\n",
      "AdjOpen       813 non-null float64\n",
      "AdjHigh       813 non-null float64\n",
      "AdjLow        813 non-null float64\n",
      "AdjClose      813 non-null float64\n",
      "AdjVolume     813 non-null float64\n",
      "dtypes: float64(12)\n",
      "memory usage: 82.6 KB\n"
     ]
    }
   ],
   "source": [
    "symbol = 'FB.US'\n",
    "\n",
    "quandl = web.DataReader(symbol, 'quandl', '2015-01-01')\n",
    "quandl.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FRED"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:15:07.662461Z",
     "start_time": "2018-12-25T19:15:07.501573Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 13 entries, 2010-01-01 to 2013-01-01\n",
      "Data columns (total 1 columns):\n",
      "GDP    13 non-null float64\n",
      "dtypes: float64(1)\n",
      "memory usage: 208.0 bytes\n"
     ]
    }
   ],
   "source": [
    "start = datetime(2010, 1, 1)\n",
    "\n",
    "end = datetime(2013, 1, 27)\n",
    "\n",
    "gdp = web.DataReader('GDP', 'fred', start, end)\n",
    "\n",
    "gdp.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:15:09.584269Z",
     "start_time": "2018-12-25T19:15:09.399866Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "DatetimeIndex: 37 entries, 2010-01-01 to 2013-01-01\n",
      "Freq: MS\n",
      "Data columns (total 2 columns):\n",
      "CPIAUCSL    37 non-null float64\n",
      "CPILFESL    37 non-null float64\n",
      "dtypes: float64(2)\n",
      "memory usage: 888.0 bytes\n"
     ]
    }
   ],
   "source": [
    "inflation = web.DataReader(['CPIAUCSL', 'CPILFESL'], 'fred', start, end)\n",
    "inflation.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fama/French"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:15:10.764850Z",
     "start_time": "2018-12-25T19:15:10.755565Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['F-F_Research_Data_Factors',\n",
       " 'F-F_Research_Data_Factors_weekly',\n",
       " 'F-F_Research_Data_Factors_daily',\n",
       " 'F-F_Research_Data_5_Factors_2x3',\n",
       " 'F-F_Research_Data_5_Factors_2x3_daily',\n",
       " 'Portfolios_Formed_on_ME',\n",
       " 'Portfolios_Formed_on_ME_Wout_Div',\n",
       " 'Portfolios_Formed_on_ME_Daily',\n",
       " 'Portfolios_Formed_on_BE-ME',\n",
       " 'Portfolios_Formed_on_BE-ME_Wout_Div',\n",
       " 'Portfolios_Formed_on_BE-ME_Daily',\n",
       " 'Portfolios_Formed_on_OP',\n",
       " 'Portfolios_Formed_on_OP_Wout_Div',\n",
       " 'Portfolios_Formed_on_INV',\n",
       " 'Portfolios_Formed_on_INV_Wout_Div',\n",
       " '6_Portfolios_2x3',\n",
       " '6_Portfolios_2x3_Wout_Div',\n",
       " '6_Portfolios_2x3_weekly',\n",
       " '6_Portfolios_2x3_daily',\n",
       " '25_Portfolios_5x5',\n",
       " '25_Portfolios_5x5_Wout_Div',\n",
       " '25_Portfolios_5x5_Daily',\n",
       " '100_Portfolios_10x10',\n",
       " '100_Portfolios_10x10_Wout_Div',\n",
       " '100_Portfolios_10x10_Daily',\n",
       " '6_Portfolios_ME_OP_2x3',\n",
       " '6_Portfolios_ME_OP_2x3_Wout_Div',\n",
       " '6_Portfolios_ME_OP_2x3_daily',\n",
       " '25_Portfolios_ME_OP_5x5',\n",
       " '25_Portfolios_ME_OP_5x5_Wout_Div',\n",
       " '25_Portfolios_ME_OP_5x5_daily',\n",
       " '100_Portfolios_ME_OP_10x10',\n",
       " '100_Portfolios_10x10_ME_OP_Wout_Div',\n",
       " '100_Portfolios_ME_OP_10x10_daily',\n",
       " '6_Portfolios_ME_INV_2x3',\n",
       " '6_Portfolios_ME_INV_2x3_Wout_Div',\n",
       " '6_Portfolios_ME_INV_2x3_daily',\n",
       " '25_Portfolios_ME_INV_5x5',\n",
       " '25_Portfolios_ME_INV_5x5_Wout_Div',\n",
       " '25_Portfolios_ME_INV_5x5_daily',\n",
       " '100_Portfolios_ME_INV_10x10',\n",
       " '100_Portfolios_10x10_ME_INV_Wout_Div',\n",
       " '100_Portfolios_ME_INV_10x10_daily',\n",
       " '25_Portfolios_BEME_OP_5x5',\n",
       " '25_Portfolios_BEME_OP_5x5_Wout_Div',\n",
       " '25_Portfolios_BEME_OP_5x5_daily',\n",
       " '25_Portfolios_BEME_INV_5x5',\n",
       " '25_Portfolios_BEME_INV_5x5_Wout_Div',\n",
       " '25_Portfolios_BEME_INV_5x5_daily',\n",
       " '25_Portfolios_OP_INV_5x5',\n",
       " '25_Portfolios_OP_INV_5x5_Wout_Div',\n",
       " '25_Portfolios_OP_INV_5x5_daily',\n",
       " '32_Portfolios_ME_BEME_OP_2x4x4',\n",
       " '32_Portfolios_ME_BEME_OP_2x4x4_Wout_Div',\n",
       " '32_Portfolios_ME_BEME_INV_2x4x4',\n",
       " '32_Portfolios_ME_BEME_INV_2x4x4_Wout_Div',\n",
       " '32_Portfolios_ME_OP_INV_2x4x4',\n",
       " '32_Portfolios_ME_OP_INV_2x4x4_Wout_Div',\n",
       " 'Portfolios_Formed_on_E-P',\n",
       " 'Portfolios_Formed_on_E-P_Wout_Div',\n",
       " 'Portfolios_Formed_on_CF-P',\n",
       " 'Portfolios_Formed_on_CF-P_Wout_Div',\n",
       " 'Portfolios_Formed_on_D-P',\n",
       " 'Portfolios_Formed_on_D-P_Wout_Div',\n",
       " '6_Portfolios_ME_EP_2x3',\n",
       " '6_Portfolios_ME_EP_2x3_Wout_Div',\n",
       " '6_Portfolios_ME_CFP_2x3',\n",
       " '6_Portfolios_ME_CFP_2x3_Wout_Div',\n",
       " '6_Portfolios_ME_DP_2x3',\n",
       " '6_Portfolios_ME_DP_2x3_Wout_Div',\n",
       " 'F-F_Momentum_Factor',\n",
       " 'F-F_Momentum_Factor_daily',\n",
       " '6_Portfolios_ME_Prior_12_2',\n",
       " '6_Portfolios_ME_Prior_12_2_Daily',\n",
       " '25_Portfolios_ME_Prior_12_2',\n",
       " '25_Portfolios_ME_Prior_12_2_Daily',\n",
       " '10_Portfolios_Prior_12_2',\n",
       " '10_Portfolios_Prior_12_2_Daily',\n",
       " 'F-F_ST_Reversal_Factor',\n",
       " 'F-F_ST_Reversal_Factor_daily',\n",
       " '6_Portfolios_ME_Prior_1_0',\n",
       " '6_Portfolios_ME_Prior_1_0_Daily',\n",
       " '25_Portfolios_ME_Prior_1_0',\n",
       " '25_Portfolios_ME_Prior_1_0_Daily',\n",
       " '10_Portfolios_Prior_1_0',\n",
       " '10_Portfolios_Prior_1_0_Daily',\n",
       " 'F-F_LT_Reversal_Factor',\n",
       " 'F-F_LT_Reversal_Factor_daily',\n",
       " '6_Portfolios_ME_Prior_60_13',\n",
       " '6_Portfolios_ME_Prior_60_13_Daily',\n",
       " '25_Portfolios_ME_Prior_60_13',\n",
       " '25_Portfolios_ME_Prior_60_13_Daily',\n",
       " '10_Portfolios_Prior_60_13',\n",
       " '10_Portfolios_Prior_60_13_Daily',\n",
       " 'Portfolios_Formed_on_AC',\n",
       " '25_Portfolios_ME_AC_5x5',\n",
       " 'Portfolios_Formed_on_BETA',\n",
       " '25_Portfolios_ME_BETA_5x5',\n",
       " 'Portfolios_Formed_on_NI',\n",
       " '25_Portfolios_ME_NI_5x5',\n",
       " 'Portfolios_Formed_on_VAR',\n",
       " '25_Portfolios_ME_VAR_5x5',\n",
       " 'Portfolios_Formed_on_RESVAR',\n",
       " '25_Portfolios_ME_RESVAR_5x5',\n",
       " '5_Industry_Portfolios',\n",
       " '5_Industry_Portfolios_Wout_Div',\n",
       " '5_Industry_Portfolios_daily',\n",
       " '10_Industry_Portfolios',\n",
       " '10_Industry_Portfolios_Wout_Div',\n",
       " '10_Industry_Portfolios_daily',\n",
       " '12_Industry_Portfolios',\n",
       " '12_Industry_Portfolios_Wout_Div',\n",
       " '12_Industry_Portfolios_daily',\n",
       " '17_Industry_Portfolios',\n",
       " '17_Industry_Portfolios_Wout_Div',\n",
       " '17_Industry_Portfolios_daily',\n",
       " '30_Industry_Portfolios',\n",
       " '30_Industry_Portfolios_Wout_Div',\n",
       " '30_Industry_Portfolios_daily',\n",
       " '38_Industry_Portfolios',\n",
       " '38_Industry_Portfolios_Wout_Div',\n",
       " '38_Industry_Portfolios_daily',\n",
       " '48_Industry_Portfolios',\n",
       " '48_Industry_Portfolios_Wout_Div',\n",
       " '48_Industry_Portfolios_daily',\n",
       " '49_Industry_Portfolios',\n",
       " '49_Industry_Portfolios_Wout_Div',\n",
       " '49_Industry_Portfolios_daily',\n",
       " 'ME_Breakpoints',\n",
       " 'BE-ME_Breakpoints',\n",
       " 'OP_Breakpoints',\n",
       " 'INV_Breakpoints',\n",
       " 'E-P_Breakpoints',\n",
       " 'CF-P_Breakpoints',\n",
       " 'D-P_Breakpoints',\n",
       " 'Prior_2-12_Breakpoints',\n",
       " 'Global_3_Factors',\n",
       " 'Global_3_Factors_Daily',\n",
       " 'Global_ex_US_3_Factors',\n",
       " 'Global_ex_US_3_Factors_Daily',\n",
       " 'Europe_3_Factors',\n",
       " 'Europe_3_Factors_Daily',\n",
       " 'Japan_3_Factors',\n",
       " 'Japan_3_Factors_Daily',\n",
       " 'Asia_Pacific_ex_Japan_3_Factors',\n",
       " 'Asia_Pacific_ex_Japan_3_Factors_Daily',\n",
       " 'North_America_3_Factors',\n",
       " 'North_America_3_Factors_Daily',\n",
       " 'Global_5_Factors',\n",
       " 'Global_5_Factors_Daily',\n",
       " 'Global_ex_US_5_Factors',\n",
       " 'Global_ex_US_5_Factors_Daily',\n",
       " 'Europe_5_Factors',\n",
       " 'Europe_5_Factors_Daily',\n",
       " 'Japan_5_Factors',\n",
       " 'Japan_5_Factors_Daily',\n",
       " 'Asia_Pacific_ex_Japan_5_Factors',\n",
       " 'Asia_Pacific_ex_Japan_5_Factors_Daily',\n",
       " 'North_America_5_Factors',\n",
       " 'North_America_5_Factors_Daily',\n",
       " 'Global_Mom_Factor',\n",
       " 'Global_Mom_Factor_Daily',\n",
       " 'Global_ex_US_Mom_Factor',\n",
       " 'Global_ex_US_Mom_Factor_Daily',\n",
       " 'Europe_Mom_Factor',\n",
       " 'Europe_Mom_Factor_Daily',\n",
       " 'Japan_Mom_Factor',\n",
       " 'Japan_Mom_Factor_Daily',\n",
       " 'Asia_Pacific_ex_Japan_MOM_Factor',\n",
       " 'Asia_Pacific_ex_Japan_MOM_Factor_Daily',\n",
       " 'North_America_Mom_Factor',\n",
       " 'North_America_Mom_Factor_Daily',\n",
       " 'Global_6_Portfolios_ME_BE-ME',\n",
       " 'Global_6_Portfolios_ME_BE-ME_daily',\n",
       " 'Global_ex_US_6_Portfolios_ME_BE-ME',\n",
       " 'Global_ex_US_6_Portfolios_ME_BE-ME_daily',\n",
       " 'Europe_6_Portfolios_ME_BE-ME',\n",
       " 'Europe_6_Portfolios_ME_BE-ME_daily',\n",
       " 'Japan_6_Portfolios_ME_BE-ME',\n",
       " 'Japan_6_Portfolios_ME_BE-ME_daily',\n",
       " 'Asia_Pacific_ex_Japan_6_Portfolios_ME_BE-ME',\n",
       " 'Asia_Pacific_ex_Japan_6_Portfolios_ME_BE-ME_daily',\n",
       " 'North_America_6_Portfolios_ME_BE-ME',\n",
       " 'North_America_6_Portfolios_ME_BE-ME_daily',\n",
       " 'Global_25_Portfolios_ME_BE-ME',\n",
       " 'Global_25_Portfolios_ME_BE-ME_daily',\n",
       " 'Global_ex_US_25_Portfolios_ME_BE-ME',\n",
       " 'Global_ex_US_25_Portfolios_ME_BE-ME_daily',\n",
       " 'Europe_25_Portfolios_ME_BE-ME',\n",
       " 'Europe_25_Portfolios_ME_BE-ME_daily',\n",
       " 'Japan_25_Portfolios_ME_BE-ME',\n",
       " 'Japan_25_Portfolios_ME_BE-ME_daily',\n",
       " 'Asia_Pacific_ex_Japan_25_Portfolios_ME_BE-ME',\n",
       " 'Asia_Pacific_ex_Japan_25_Portfolios_ME_BE-ME_daily',\n",
       " 'North_America_25_Portfolios_ME_BE-ME',\n",
       " 'North_America_25_Portfolios_ME_BE-ME_daily',\n",
       " 'Global_6_Portfolios_ME_OP',\n",
       " 'Global_6_Portfolios_ME_OP_Daily',\n",
       " 'Global_ex_US_6_Portfolios_ME_OP',\n",
       " 'Global_ex_US_6_Portfolios_ME_OP_Daily',\n",
       " 'Europe_6_Portfolios_ME_OP',\n",
       " 'Europe_6_Portfolios_ME_OP_Daily',\n",
       " 'Japan_6_Portfolios_ME_OP',\n",
       " 'Japan_6_Portfolios_ME_OP_Daily',\n",
       " 'Asia_Pacific_ex_Japan_6_Portfolios_ME_OP',\n",
       " 'Asia_Pacific_ex_Japan_6_Portfolios_ME_OP_Daily',\n",
       " 'North_America_6_Portfolios_ME_OP',\n",
       " 'North_America_6_Portfolios_ME_OP_Daily',\n",
       " 'Global_25_Portfolios_ME_OP',\n",
       " 'Global_25_Portfolios_ME_OP_Daily',\n",
       " 'Global_ex_US_25_Portfolios_ME_OP',\n",
       " 'Global_ex_US_25_Portfolios_ME_OP_Daily',\n",
       " 'Europe_25_Portfolios_ME_OP',\n",
       " 'Europe_25_Portfolios_ME_OP_Daily',\n",
       " 'Japan_25_Portfolios_ME_OP',\n",
       " 'Japan_25_Portfolios_ME_OP_Daily',\n",
       " 'Asia_Pacific_ex_Japan_25_Portfolios_ME_OP',\n",
       " 'Asia_Pacific_ex_Japan_25_Portfolios_ME_OP_Daily',\n",
       " 'North_America_25_Portfolios_ME_OP',\n",
       " 'North_America_25_Portfolios_ME_OP_Daily',\n",
       " 'Global_6_Portfolios_ME_INV',\n",
       " 'Global_6_Portfolios_ME_INV_Daily',\n",
       " 'Global_ex_US_6_Portfolios_ME_INV',\n",
       " 'Global_ex_US_6_Portfolios_ME_INV_Daily',\n",
       " 'Europe_6_Portfolios_ME_INV',\n",
       " 'Europe_6_Portfolios_ME_INV_Daily',\n",
       " 'Japan_6_Portfolios_ME_INV',\n",
       " 'Japan_6_Portfolios_ME_INV_Daily',\n",
       " 'Asia_Pacific_ex_Japan_6_Portfolios_ME_INV',\n",
       " 'Asia_Pacific_ex_Japan_6_Portfolios_ME_INV_Daily',\n",
       " 'North_America_6_Portfolios_ME_INV',\n",
       " 'North_America_6_Portfolios_ME_INV_Daily',\n",
       " 'Global_25_Portfolios_ME_INV',\n",
       " 'Global_25_Portfolios_ME_INV_Daily',\n",
       " 'Global_ex_US_25_Portfolios_ME_INV',\n",
       " 'Global_ex_US_25_Portfolios_ME_INV_Daily',\n",
       " 'Europe_25_Portfolios_ME_INV',\n",
       " 'Europe_25_Portfolios_ME_INV_Daily',\n",
       " 'Japan_25_Portfolios_ME_INV',\n",
       " 'Japan_25_Portfolios_ME_INV_Daily',\n",
       " 'Asia_Pacific_ex_Japan_25_Portfolios_ME_INV',\n",
       " 'Asia_Pacific_ex_Japan_25_Portfolios_ME_INV_Daily',\n",
       " 'North_America_25_Portfolios_ME_INV',\n",
       " 'North_America_25_Portfolios_ME_INV_Daily',\n",
       " 'Global_6_Portfolios_ME_Prior_12_2',\n",
       " 'Global_6_Portfolios_ME_Prior_250_20_daily',\n",
       " 'Global_ex_US_6_Portfolios_ME_Prior_12_2',\n",
       " 'Global_ex_US_6_Portfolios_ME_Prior_250_20_daily',\n",
       " 'Europe_6_Portfolios_ME_Prior_12_2',\n",
       " 'Europe_6_Portfolios_ME_Prior_250_20_daily',\n",
       " 'Japan_6_Portfolios_ME_Prior_12_2',\n",
       " 'Japan_6_Portfolios_ME_Prior_250_20_daily',\n",
       " 'Asia_Pacific_ex_Japan_6_Portfolios_ME_Prior_12_2',\n",
       " 'Asia_Pacific_ex_Japan_6_Portfolios_ME_Prior_250_20_daily',\n",
       " 'North_America_6_Portfolios_ME_Prior_12_2',\n",
       " 'North_America_6_Portfolios_ME_Prior_250_20_daily',\n",
       " 'Global_25_Portfolios_ME_Prior_12_2',\n",
       " 'Global_25_Portfolios_ME_Prior_250_20_daily',\n",
       " 'Global_ex_US_25_Portfolios_ME_Prior_12_2',\n",
       " 'Global_ex_US_25_Portfolios_ME_Prior_250_20_daily',\n",
       " 'Europe_25_Portfolios_ME_Prior_12_2',\n",
       " 'Europe_25_Portfolios_ME_Prior_250_20_daily',\n",
       " 'Japan_25_Portfolios_ME_Prior_12_2',\n",
       " 'Japan_25_Portfolios_ME_Prior_250_20_daily',\n",
       " 'Asia_Pacific_ex_Japan_25_Portfolios_ME_Prior_12_2',\n",
       " 'Asia_Pacific_ex_Japan_25_Portfolios_ME_Prior_250_20_daily',\n",
       " 'North_America_25_Portfolios_ME_Prior_12_2',\n",
       " 'North_America_25_Portfolios_ME_Prior_250_20_daily',\n",
       " 'Global_32_Portfolios_ME_BE-ME_OP_2x4x4',\n",
       " 'Global_ex_US_32_Portfolios_ME_BE-ME_OP_2x4x4',\n",
       " 'Europe_32_Portfolios_ME_BE-ME_OP_2x4x4',\n",
       " 'Japan_32_Portfolios_ME_BE-ME_OP_2x4x4',\n",
       " 'Asia_Pacific_ex_Japan_32_Portfolios_ME_BE-ME_OP_2x4x4',\n",
       " 'North_America_32_Portfolios_ME_BE-ME_OP_2x4x4',\n",
       " 'Global_32_Portfolios_ME_BE-ME_INV(TA)_2x4x4',\n",
       " 'Global_ex_US_32_Portfolios_ME_BE-ME_INV(TA)_2x4x4',\n",
       " 'Europe_32_Portfolios_ME_BE-ME_INV(TA)_2x4x4',\n",
       " 'Japan_32_Portfolios_ME_BE-ME_INV(TA)_2x4x4',\n",
       " 'Asia_Pacific_ex_Japan_32_Portfolios_ME_BE-ME_INV(TA)_2x4x4',\n",
       " 'North_America_32_Portfolios_ME_BE-ME_INV(TA)_2x4x4',\n",
       " 'Global_32_Portfolios_ME_INV(TA)_OP_2x4x4',\n",
       " 'Global_ex_US_32_Portfolios_ME_INV(TA)_OP_2x4x4',\n",
       " 'Europe_32_Portfolios_ME_INV(TA)_OP_2x4x4',\n",
       " 'Japan_32_Portfolios_ME_INV(TA)_OP_2x4x4',\n",
       " 'Asia_Pacific_ex_Japan_32_Portfolios_ME_INV(TA)_OP_2x4x4',\n",
       " 'North_America_32_Portfolios_ME_INV(TA)_OP_2x4x4']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pandas_datareader.famafrench import get_available_datasets\n",
    "get_available_datasets()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:16:23.570899Z",
     "start_time": "2018-12-25T19:16:23.299817Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5 Industry Portfolios\n",
      "---------------------\n",
      "\n",
      "This file was created by CMPT_IND_RETS using the 201901 CRSP database. It contains value- and equal-weighted returns for 5 industry portfolios. The portfolios are constructed at the end of June. The annual returns are from January to December. Missing data are indicated by -99.99 or -999. Copyright 2019 Kenneth R. French\n",
      "\n",
      "  0 : Average Value Weighted Returns -- Monthly (109 rows x 5 cols)\n",
      "  1 : Average Equal Weighted Returns -- Monthly (109 rows x 5 cols)\n",
      "  2 : Average Value Weighted Returns -- Annual (9 rows x 5 cols)\n",
      "  3 : Average Equal Weighted Returns -- Annual (9 rows x 5 cols)\n",
      "  4 : Number of Firms in Portfolios (109 rows x 5 cols)\n",
      "  5 : Average Firm Size (109 rows x 5 cols)\n",
      "  6 : Sum of BE / Sum of ME (9 rows x 5 cols)\n",
      "  7 : Value-Weighted Average of BE/ME (9 rows x 5 cols)\n"
     ]
    }
   ],
   "source": [
    "ds = web.DataReader('5_Industry_Portfolios', 'famafrench')\n",
    "print(ds['DESCR'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### World Bank"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>name</th>\n",
       "      <th>source</th>\n",
       "      <th>sourceNote</th>\n",
       "      <th>sourceOrganization</th>\n",
       "      <th>topics</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>646</th>\n",
       "      <td>6.0.GDPpc_constant</td>\n",
       "      <td>GDP per capita, PPP (constant 2011 internation...</td>\n",
       "      <td>LAC Equity Lab</td>\n",
       "      <td>GDP per capita based on purchasing power parit...</td>\n",
       "      <td>b'World Development Indicators (World Bank)'</td>\n",
       "      <td>Economy &amp; Growth</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9108</th>\n",
       "      <td>NY.GDP.PCAP.KD</td>\n",
       "      <td>GDP per capita (constant 2010 US$)</td>\n",
       "      <td>World Development Indicators</td>\n",
       "      <td>GDP per capita is gross domestic product divid...</td>\n",
       "      <td>b'World Bank national accounts data, and OECD ...</td>\n",
       "      <td>Economy &amp; Growth</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9110</th>\n",
       "      <td>NY.GDP.PCAP.KN</td>\n",
       "      <td>GDP per capita (constant LCU)</td>\n",
       "      <td>World Development Indicators</td>\n",
       "      <td>GDP per capita is gross domestic product divid...</td>\n",
       "      <td>b'World Bank national accounts data, and OECD ...</td>\n",
       "      <td>Economy &amp; Growth</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9112</th>\n",
       "      <td>NY.GDP.PCAP.PP.KD</td>\n",
       "      <td>GDP per capita, PPP (constant 2011 internation...</td>\n",
       "      <td>World Development Indicators</td>\n",
       "      <td>GDP per capita based on purchasing power parit...</td>\n",
       "      <td>b'World Bank, International Comparison Program...</td>\n",
       "      <td>Economy &amp; Growth</td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9113</th>\n",
       "      <td>NY.GDP.PCAP.PP.KD.87</td>\n",
       "      <td>GDP per capita, PPP (constant 1987 internation...</td>\n",
       "      <td>WDI Database Archives</td>\n",
       "      <td></td>\n",
       "      <td>b''</td>\n",
       "      <td></td>\n",
       "      <td></td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        id                                               name  \\\n",
       "646     6.0.GDPpc_constant  GDP per capita, PPP (constant 2011 internation...   \n",
       "9108        NY.GDP.PCAP.KD                 GDP per capita (constant 2010 US$)   \n",
       "9110        NY.GDP.PCAP.KN                      GDP per capita (constant LCU)   \n",
       "9112     NY.GDP.PCAP.PP.KD  GDP per capita, PPP (constant 2011 internation...   \n",
       "9113  NY.GDP.PCAP.PP.KD.87  GDP per capita, PPP (constant 1987 internation...   \n",
       "\n",
       "                            source  \\\n",
       "646                 LAC Equity Lab   \n",
       "9108  World Development Indicators   \n",
       "9110  World Development Indicators   \n",
       "9112  World Development Indicators   \n",
       "9113         WDI Database Archives   \n",
       "\n",
       "                                             sourceNote  \\\n",
       "646   GDP per capita based on purchasing power parit...   \n",
       "9108  GDP per capita is gross domestic product divid...   \n",
       "9110  GDP per capita is gross domestic product divid...   \n",
       "9112  GDP per capita based on purchasing power parit...   \n",
       "9113                                                      \n",
       "\n",
       "                                     sourceOrganization            topics unit  \n",
       "646        b'World Development Indicators (World Bank)'  Economy & Growth       \n",
       "9108  b'World Bank national accounts data, and OECD ...  Economy & Growth       \n",
       "9110  b'World Bank national accounts data, and OECD ...  Economy & Growth       \n",
       "9112  b'World Bank, International Comparison Program...  Economy & Growth       \n",
       "9113                                                b''                         "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pandas_datareader import wb\n",
    "gdp_variables = wb.search('gdp.*capita.*const')\n",
    "gdp_variables.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>NY.GDP.PCAP.KD</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>country</th>\n",
       "      <th>year</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">Canada</th>\n",
       "      <th>2018</th>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2017</th>\n",
       "      <td>51315.888975</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2016</th>\n",
       "      <td>50407.341330</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015</th>\n",
       "      <td>50303.836848</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014</th>\n",
       "      <td>50221.841982</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              NY.GDP.PCAP.KD\n",
       "country year                \n",
       "Canada  2018             NaN\n",
       "        2017    51315.888975\n",
       "        2016    50407.341330\n",
       "        2015    50303.836848\n",
       "        2014    50221.841982"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wb_data = wb.download(indicator='NY.GDP.PCAP.KD', \n",
    "                      country=['US', 'CA', 'MX'], \n",
    "                      start=1990, \n",
    "                      end=2019)\n",
    "wb_data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### OECD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>Country</th>\n",
       "      <th colspan=\"10\" halign=\"left\">Japan</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"10\" halign=\"left\">United States</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Frequency</th>\n",
       "      <th colspan=\"10\" halign=\"left\">Annual</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"10\" halign=\"left\">Annual</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Source</th>\n",
       "      <th colspan=\"6\" halign=\"left\">Survey data</th>\n",
       "      <th colspan=\"4\" halign=\"left\">Administrative data</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"4\" halign=\"left\">Survey data</th>\n",
       "      <th colspan=\"6\" halign=\"left\">Administrative data</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Series</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Union members</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Trade union  density</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Employees</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Union members</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Trade union  density</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Trade union  density</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Employees</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Union members</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Trade union  density</th>\n",
       "      <th colspan=\"2\" halign=\"left\">Employees</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Measure</th>\n",
       "      <th>Thousands</th>\n",
       "      <th>Percentage</th>\n",
       "      <th>Thousands</th>\n",
       "      <th>Percentage</th>\n",
       "      <th>Thousands</th>\n",
       "      <th>Percentage</th>\n",
       "      <th>Thousands</th>\n",
       "      <th>Percentage</th>\n",
       "      <th>Thousands</th>\n",
       "      <th>Percentage</th>\n",
       "      <th>...</th>\n",
       "      <th>Thousands</th>\n",
       "      <th>Percentage</th>\n",
       "      <th>Thousands</th>\n",
       "      <th>Percentage</th>\n",
       "      <th>Thousands</th>\n",
       "      <th>Percentage</th>\n",
       "      <th>Thousands</th>\n",
       "      <th>Percentage</th>\n",
       "      <th>Thousands</th>\n",
       "      <th>Percentage</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Year</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2010-01-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12417.527</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>28.871256</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17.448617</td>\n",
       "      <td>97406.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2011-01-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12271.909</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>27.589723</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16.516118</td>\n",
       "      <td>102403.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2012-01-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12227.073</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>25.899329</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.861734</td>\n",
       "      <td>106924.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12396.592</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>24.489514</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15.510448</td>\n",
       "      <td>107102.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-01-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>7661.568</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>32.164433</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>17049.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>30.897064</td>\n",
       "      <td>55180.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2015-01-01</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>8359.876</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>34.516416</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>16303.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>29.518378</td>\n",
       "      <td>55230.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "Country            Japan                                                       \\\n",
       "Frequency         Annual                                                        \n",
       "Source       Survey data                                                        \n",
       "Series     Union members            Trade union  density            Employees   \n",
       "Measure        Thousands Percentage            Thousands Percentage Thousands   \n",
       "Year                                                                            \n",
       "2010-01-01           NaN        NaN                  NaN        NaN       NaN   \n",
       "2011-01-01           NaN        NaN                  NaN        NaN       NaN   \n",
       "2012-01-01           NaN        NaN                  NaN        NaN       NaN   \n",
       "2013-01-01           NaN        NaN                  NaN        NaN       NaN   \n",
       "2014-01-01           NaN        NaN                  NaN        NaN       NaN   \n",
       "2015-01-01           NaN        NaN                  NaN        NaN       NaN   \n",
       "\n",
       "Country                                                                    \\\n",
       "Frequency                                                                   \n",
       "Source                Administrative data                                   \n",
       "Series                      Union members            Trade union  density   \n",
       "Measure    Percentage           Thousands Percentage            Thousands   \n",
       "Year                                                                        \n",
       "2010-01-01        NaN           12417.527        NaN                  NaN   \n",
       "2011-01-01        NaN           12271.909        NaN                  NaN   \n",
       "2012-01-01        NaN           12227.073        NaN                  NaN   \n",
       "2013-01-01        NaN           12396.592        NaN                  NaN   \n",
       "2014-01-01        NaN            7661.568        NaN                  NaN   \n",
       "2015-01-01        NaN            8359.876        NaN                  NaN   \n",
       "\n",
       "Country                ...        United States                       \\\n",
       "Frequency              ...               Annual                        \n",
       "Source                 ...          Survey data                        \n",
       "Series                 ... Trade union  density            Employees   \n",
       "Measure    Percentage  ...            Thousands Percentage Thousands   \n",
       "Year                   ...                                             \n",
       "2010-01-01  28.871256  ...                  NaN  17.448617   97406.0   \n",
       "2011-01-01  27.589723  ...                  NaN  16.516118  102403.0   \n",
       "2012-01-01  25.899329  ...                  NaN  15.861734  106924.0   \n",
       "2013-01-01  24.489514  ...                  NaN  15.510448  107102.0   \n",
       "2014-01-01  32.164433  ...                  NaN        NaN       NaN   \n",
       "2015-01-01  34.516416  ...                  NaN        NaN       NaN   \n",
       "\n",
       "Country                                                                    \\\n",
       "Frequency                                                                   \n",
       "Source                Administrative data                                   \n",
       "Series                      Union members            Trade union  density   \n",
       "Measure    Percentage           Thousands Percentage            Thousands   \n",
       "Year                                                                        \n",
       "2010-01-01        NaN                 NaN        NaN                  NaN   \n",
       "2011-01-01        NaN                 NaN        NaN                  NaN   \n",
       "2012-01-01        NaN                 NaN        NaN                  NaN   \n",
       "2013-01-01        NaN                 NaN        NaN                  NaN   \n",
       "2014-01-01        NaN             17049.0        NaN                  NaN   \n",
       "2015-01-01        NaN             16303.0        NaN                  NaN   \n",
       "\n",
       "Country                                     \n",
       "Frequency                                   \n",
       "Source                                      \n",
       "Series                Employees             \n",
       "Measure    Percentage Thousands Percentage  \n",
       "Year                                        \n",
       "2010-01-01        NaN       NaN        NaN  \n",
       "2011-01-01        NaN       NaN        NaN  \n",
       "2012-01-01        NaN       NaN        NaN  \n",
       "2013-01-01        NaN       NaN        NaN  \n",
       "2014-01-01  30.897064   55180.0        NaN  \n",
       "2015-01-01  29.518378   55230.0        NaN  \n",
       "\n",
       "[6 rows x 24 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = web.DataReader('TUD', 'oecd', end='2015')\n",
    "df[['Japan', 'United States']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### EuroStat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = web.DataReader('tran_sf_railac', 'eurostat')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>ACCIDENT</th>\n",
       "      <th colspan=\"10\" halign=\"left\">Collisions of trains, including collisions with obstacles within the clearance gauge</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"10\" halign=\"left\">Unknown</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIT</th>\n",
       "      <th colspan=\"10\" halign=\"left\">Number</th>\n",
       "      <th>...</th>\n",
       "      <th colspan=\"10\" halign=\"left\">Number</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GEO</th>\n",
       "      <th>Austria</th>\n",
       "      <th>Belgium</th>\n",
       "      <th>Bulgaria</th>\n",
       "      <th>Switzerland</th>\n",
       "      <th>Channel Tunnel</th>\n",
       "      <th>Czechia</th>\n",
       "      <th>Germany (until 1990 former territory of the FRG)</th>\n",
       "      <th>Denmark</th>\n",
       "      <th>Estonia</th>\n",
       "      <th>Greece</th>\n",
       "      <th>...</th>\n",
       "      <th>Netherlands</th>\n",
       "      <th>Norway</th>\n",
       "      <th>Poland</th>\n",
       "      <th>Portugal</th>\n",
       "      <th>Romania</th>\n",
       "      <th>Sweden</th>\n",
       "      <th>Slovenia</th>\n",
       "      <th>Slovakia</th>\n",
       "      <th>Turkey</th>\n",
       "      <th>United Kingdom</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>FREQ</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>...</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "      <th>Annual</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TIME_PERIOD</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2010-01-01</th>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2011-01-01</th>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>18.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2012-01-01</th>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>23.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2013-01-01</th>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>29.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2014-01-01</th>\n",
       "      <td>1.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>32.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 264 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "ACCIDENT    Collisions of trains, including collisions with obstacles within the clearance gauge  \\\n",
       "UNIT                                                                                      Number   \n",
       "GEO                                                                                      Austria   \n",
       "FREQ                                                                                      Annual   \n",
       "TIME_PERIOD                                                                                        \n",
       "2010-01-01                                                 3.0                                     \n",
       "2011-01-01                                                 2.0                                     \n",
       "2012-01-01                                                 1.0                                     \n",
       "2013-01-01                                                 4.0                                     \n",
       "2014-01-01                                                 1.0                                     \n",
       "\n",
       "ACCIDENT                                                         \\\n",
       "UNIT                                                              \n",
       "GEO         Belgium Bulgaria Switzerland Channel Tunnel Czechia   \n",
       "FREQ         Annual   Annual      Annual         Annual  Annual   \n",
       "TIME_PERIOD                                                       \n",
       "2010-01-01      5.0      2.0         5.0            0.0     3.0   \n",
       "2011-01-01      0.0      0.0         4.0            0.0     6.0   \n",
       "2012-01-01      3.0      3.0         4.0            0.0     6.0   \n",
       "2013-01-01      1.0      2.0         6.0            0.0     5.0   \n",
       "2014-01-01      3.0      4.0         0.0            0.0    13.0   \n",
       "\n",
       "ACCIDENT                                                                      \\\n",
       "UNIT                                                                           \n",
       "GEO         Germany (until 1990 former territory of the FRG) Denmark Estonia   \n",
       "FREQ                                                  Annual  Annual  Annual   \n",
       "TIME_PERIOD                                                                    \n",
       "2010-01-01                                              13.0     0.0     1.0   \n",
       "2011-01-01                                              18.0     1.0     0.0   \n",
       "2012-01-01                                              23.0     1.0     3.0   \n",
       "2013-01-01                                              29.0     0.0     0.0   \n",
       "2014-01-01                                              32.0     0.0     0.0   \n",
       "\n",
       "ACCIDENT            ...     Unknown                                        \\\n",
       "UNIT                ...      Number                                         \n",
       "GEO         Greece  ... Netherlands Norway Poland Portugal Romania Sweden   \n",
       "FREQ        Annual  ...      Annual Annual Annual   Annual  Annual Annual   \n",
       "TIME_PERIOD         ...                                                     \n",
       "2010-01-01     4.0  ...         NaN    NaN    NaN      NaN     NaN    NaN   \n",
       "2011-01-01     1.0  ...         NaN    NaN    NaN      NaN     NaN    NaN   \n",
       "2012-01-01     2.0  ...         NaN    NaN    NaN      NaN     NaN    NaN   \n",
       "2013-01-01     2.0  ...         NaN    NaN    NaN      NaN     NaN    NaN   \n",
       "2014-01-01     1.0  ...         NaN    NaN    NaN      NaN     NaN    NaN   \n",
       "\n",
       "ACCIDENT                                             \n",
       "UNIT                                                 \n",
       "GEO         Slovenia Slovakia Turkey United Kingdom  \n",
       "FREQ          Annual   Annual Annual         Annual  \n",
       "TIME_PERIOD                                          \n",
       "2010-01-01       NaN      NaN    0.0            NaN  \n",
       "2011-01-01       NaN      NaN    0.0            NaN  \n",
       "2012-01-01       NaN      NaN    0.0            NaN  \n",
       "2013-01-01       NaN      NaN    0.0            NaN  \n",
       "2014-01-01       NaN      NaN    0.0            NaN  \n",
       "\n",
       "[5 rows x 264 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "### Stooq"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Google finance stopped providing common index data download. The Stooq site had this data for download for a while but is currently broken, awaiting release of [fix](https://github.com/pydata/pandas-datareader/issues/594)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:17:11.052846Z",
     "start_time": "2018-12-25T19:17:08.631634Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "46"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "index_url = 'https://stooq.com/t/'\n",
    "ix = pd.read_html(index_url)\n",
    "len(ix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:17:11.497938Z",
     "start_time": "2018-12-25T19:17:11.054004Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 0 entries\n",
      "Empty DataFrame"
     ]
    }
   ],
   "source": [
    "f = web.DataReader('^SPX', 'stooq', start='20000101')\n",
    "f.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:17:11.521392Z",
     "start_time": "2018-12-25T19:17:11.503799Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>No data</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: []"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NASDAQ Symbols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:20:56.673795Z",
     "start_time": "2018-12-25T19:20:55.625622Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Index: 8701 entries, A to ZYXI\n",
      "Data columns (total 11 columns):\n",
      "Nasdaq Traded       8701 non-null bool\n",
      "Security Name       8701 non-null object\n",
      "Listing Exchange    8701 non-null category\n",
      "Market Category     8701 non-null object\n",
      "ETF                 8701 non-null bool\n",
      "Round Lot Size      8701 non-null float64\n",
      "Test Issue          8701 non-null bool\n",
      "Financial Status    3411 non-null category\n",
      "CQS Symbol          5290 non-null object\n",
      "NASDAQ Symbol       8701 non-null object\n",
      "NextShares          8701 non-null bool\n",
      "dtypes: bool(4), category(2), float64(1), object(4)\n",
      "memory usage: 459.2+ KB\n"
     ]
    }
   ],
   "source": [
    "from pandas_datareader.nasdaq_trader import get_nasdaq_symbols\n",
    "symbols = get_nasdaq_symbols()\n",
    "symbols.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:20:59.748392Z",
     "start_time": "2018-12-25T19:20:58.646206Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "url = 'https://www.nasdaq.com/screening/companies-by-industry.aspx?exchange=NASDAQ'\n",
    "res = pd.read_html(url)\n",
    "len(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:21:02.385427Z",
     "start_time": "2018-12-25T19:21:02.352487Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1 entries, 0 to 0\n",
      "Data columns (total 2 columns):\n",
      "0    1 non-null object\n",
      "1    1 non-null object\n",
      "dtypes: object(2)\n",
      "memory usage: 96.0+ bytes\n",
      "None\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 101 entries, 0 to 100\n",
      "Data columns (total 6 columns):\n",
      "Name          101 non-null object\n",
      "Symbol        51 non-null object\n",
      "Market Cap    47 non-null object\n",
      "Country       51 non-null object\n",
      "IPO Year      28 non-null object\n",
      "Subsector     51 non-null object\n",
      "dtypes: object(6)\n",
      "memory usage: 4.8+ KB\n",
      "None\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1 entries, 0 to 0\n",
      "Data columns (total 1 columns):\n",
      "0    1 non-null object\n",
      "dtypes: object(1)\n",
      "memory usage: 88.0+ bytes\n",
      "None\n",
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1 entries, 0 to 0\n",
      "Data columns (total 1 columns):\n",
      "0    1 non-null object\n",
      "dtypes: object(1)\n",
      "memory usage: 88.0+ bytes\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "for r in res:\n",
    "    print(r.info())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tiingo"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Requires [signing up](https://api.tiingo.com/) and storing API key in environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:21:26.396222Z",
     "start_time": "2018-12-25T19:21:26.150734Z"
    }
   },
   "outputs": [],
   "source": [
    "df = web.get_data_tiingo('GOOG', api_key=os.getenv('TIINGO_API_KEY'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2018-12-25T19:21:27.003018Z",
     "start_time": "2018-12-25T19:21:26.986133Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "MultiIndex: 1244 entries, (GOOG, 2014-03-27 00:00:00) to (GOOG, 2019-03-06 00:00:00)\n",
      "Data columns (total 12 columns):\n",
      "adjClose       1244 non-null float64\n",
      "adjHigh        1244 non-null float64\n",
      "adjLow         1244 non-null float64\n",
      "adjOpen        1244 non-null float64\n",
      "adjVolume      1244 non-null int64\n",
      "close          1244 non-null float64\n",
      "divCash        1244 non-null float64\n",
      "high           1244 non-null float64\n",
      "low            1244 non-null float64\n",
      "open           1244 non-null float64\n",
      "splitFactor    1244 non-null float64\n",
      "volume         1244 non-null int64\n",
      "dtypes: float64(10), int64(2)\n",
      "memory usage: 130.1+ KB\n"
     ]
    }
   ],
   "source": [
    "df.info()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": true,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
